# -*- encoding:utf-8 -*-
import os 


def load_stopwords(path):
    try:
        with open(path,encoding='utf-8') as f:
            words = f.read().split('\n')
        f.close()
    except:
        # 
        return ['a','the']
    return words

def filter_keys(keywords:str , stopwords:str):
    """
        定义好了停词列表
    """
    keywords = keywords.split('##')
    new_keys = []
    for key in keywords:
        if key not in stopwords:
            new_keys.append(key)
    return "##".join(new_keys)


def filter_document(document):
    """
        去除一些无用的字符，后续可以增加
    """
    punctuation = '！“”#¥&：/……～+-\t\n'
    for p in punctuation:
        document = document.replace(p, '')
    return document



def parse_txt(file_name):
    """
        读取txt文件的文档内容以字符串的形式返回
    """
    try:
        with open(file_name ,'r' ,encoding='utf-8') as f:
            document  = f.read()
        f.close()
    except:
        document= "  "
    content = filter_document(document)

    return content

def parse_document(file_name):
    """
        将docx的文字内容以字符串的形式返回
        docx里面的图片是走其他的另外的image流程
    """
    try:
        from docx import Document
        document = Document(file_name)
        all_paragraphs = document.paragraphs

        doc_str = ''
        for paragraph in all_paragraphs:
            doc_str += paragraph.text
            # print(paragraph.text)
    except:
        doc_str = ''
    
    content = filter_document(doc_str)
    return content

      
def parse_pdf(file_name):
    """
        将pdf的内容以字符串的形式进行返回
        pdf里面的图片是走其他的另外的image流程
    """
    try:
        from io import StringIO
        from pdfminer.converter import TextConverter 
        from pdfminer.layout import LAParams
        from pdfminer.pdfdocument import PDFDocument
        from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
        from pdfminer.pdfpage import PDFPage
        from pdfminer.pdfparser import PDFParser

        output_string = StringIO()
        with open(file_name, 'rb' ) as in_file:
            parser = PDFParser(in_file)
            doc = PDFDocument(parser)
            rsrcmgr = PDFResourceManager()
            #  to convert it to html and use layout analysis:
            device = TextConverter(rsrcmgr, output_string, laparams=LAParams())
            interpreter = PDFPageInterpreter(rsrcmgr, device)
            for page in PDFPage.create_pages(doc):
                interpreter.process_page(page)
    except:
        return ''
    
    content = filter_document(output_string.getvalue())
    return content
    
